import numpy as np

import pandas as pd

​

from scipy import stats

from scipy.optimize import curve_fit

​

from matplotlib_inline.backend_inline import set_matplotlib_formats

import plotly.io as pio

import plotly.express as px

from plotly.subplots import make_subplots

import plotly.figure_factory as ff

​

from sklearn.model_selection import train_test_split, ShuffleSplit

from sklearn.metrics import r2_score, mean_absolute_error

from sklearn.inspection import permutation_importance

​

from catboost import Pool, CatBoostRegressor, cv

​

import shap

​

from ipywidgets import interact, interact_manual

import ipywidgets as widgets

np.random.seed(42)

pd.set_option('display.float_format', '{:.2f}'.format)

set_matplotlib_formats('svg')

pio.templates.default = 'plotly_white'

pio.templates['plotly_white']['layout']['font'] = {'color': '#2a3f5f', 'size': 14}

shap.initjs()

data = pd.read_csv('https://github.com/rusmux/yandex-games/blob/main/games_data.csv?raw=true')

data

data.columns = data.columns.str.lower()

data = data.replace('tbd', np.nan).apply(pd.to_numeric, downcast='float', errors='ignore')

data['decade_of_release'] = pd.cut(data['year_of_release'], range(1979, 2029, 10),

                                   labels=['1980s', '1990s', '2000s', '2010s'])

data['total_sales'] = data['na_sales'] + data['eu_sales'] + data['jp_sales'] + data['other_sales']

data.info(memory_usage='deep')

data.isna().agg(['sum', 'mean']).T.rename_axis('missing values', axis=1)

nan_mean = lambda x: x.isna().mean()

​

nan_by_decade = data.pivot_table(index='decade_of_release',

                                 values=['user_score', 'critic_score', 'rating'],

                                 aggfunc=[nan_mean]).rename({'<lambda>': 'mean_nan'}, axis=1, level=0)

nan_by_decade['total_games'] = data['decade_of_release'].value_counts()

nan_by_decade

data.duplicated().sum()

assert data['platform'].str.lower().nunique() == data['platform'].nunique()

assert data['name'].str.lower().nunique() == data['name'].nunique()

assert data['genre'].str.lower().nunique() == data['genre'].nunique()

assert data['rating'].str.lower().nunique() == data['rating'].nunique()

print('No implicit duplicates')

data.duplicated(subset=['name', 'platform']).sum()

data[data.duplicated(subset=['name', 'platform'])]

data.query("name == 'Need for Speed: Most Wanted' and platform in ['X360', 'PC']")

data.drop([1190, 11715], inplace=True)

data.query("name == 'Madden NFL 13' and platform == 'PS3'")

data.query("name == 'Sonic the Hedgehog' and platform == 'PS3'")

data[data['name'].isna()]

data.loc[604, 'eu_sales'] += data.loc[16230, 'eu_sales']

data.loc[1745, 'eu_sales'] += data.loc[4127, 'eu_sales']

data.drop([4127, 16230, 659, 14244], inplace=True)

print('Value counts by column')

interact(lambda column: data[column].value_counts(normalize=True, dropna=False), column=['platform', 'genre', 'rating', 'decade_of_release']);

print('Unique games:', data['name'].nunique())

data['name'].value_counts()[:20]

game_counts = data['name'].value_counts(dropna=False)

data['n_platforms'] = game_counts[data['name']].values

data['rating'].fillna('No rating', inplace=True)

data.describe(datetime_is_numeric=True)

outliers_idxs = data.query("na_sales > 2 or eu_sales > 1 or jp_sales > 1 or other_sales > 1").index

print(f"Data deleted: < {100*len(outliers_idxs)/len(data):.2f}% ({len(outliers_idxs)}/{len(data)})")

data.drop(outliers_idxs, inplace=True)

data[['na_sales', 'eu_sales', 'jp_sales', 'other_sales', 'total_sales']] *= 1000

def cat_feature_histograms(data, years, platforms):

    data = data.query('@years[1] >= year_of_release >= @years[0]')

    top_platforms = data.pivot_table(index='platform', values='total_sales', aggfunc='sum')

    top_platforms = top_platforms.sort_values(by='total_sales', ascending=False)[:platforms].index

    top_platforms_data = data.query("platform in @top_platforms")

​

    fig = px.histogram(top_platforms_data, y='platform')

​

    buttons = []

    for column in ['platform', 'genre', 'rating', 'decade_of_release']:

      button=dict(

          args=[{'y': [top_platforms_data[column] if column == 'platform' else data[column]]},

                {'yaxis': {'categoryorder': 'total ascending', 'title': column}}],

          label=column,

          method='update',

      )

      buttons.append(button)

​

    menu = dict(

        buttons=buttons,

        direction='down',

        x=-0.13,

    )

​

    fig.update_layout(

        updatemenus=[menu],

        yaxis_categoryorder='total ascending',

        title='Feature histograms',

    )

    return fig

# platforms slider is only for "platform" option

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

platforms_slider = widgets.IntSlider(value=10, min=1, max=31, continuous_update=False)

interact(cat_feature_histograms, data=widgets.fixed(data), years=years_slider, platforms=platforms_slider);

ratings_to_drop = ['EC', 'K-A', 'RP', 'AO']

platforms_to_drop = data['platform'].value_counts()[data['platform'].value_counts() < 100].index

idxs_to_drop = data.query("rating in @ratings_to_drop or platform in @platforms_to_drop").index

data.query('platform in @platforms_to_drop').pivot_table(index='platform', values='year_of_release', aggfunc='max').rename(lambda x: 'max_' + x, axis=1)

data.query('rating in @ratings_to_drop').pivot_table(index='rating', values='year_of_release', aggfunc='max').rename(lambda x: 'max_' + x, axis=1)

print(f"Data deleted: < {100*len(idxs_to_drop)/len(data):.2f}% ({len(idxs_to_drop)}/{len(data)})")

data = data.drop(idxs_to_drop).reset_index(drop=True)

def num_feature_histograms(data, years):

    data = data.query('@years[1] >= year_of_release >= @years[0]')

​

    fig = px.histogram(data, x='total_sales', marginal='box')

    columns_to_show = ['total_sales', 'na_sales', 'eu_sales', 'jp_sales', 'other_sales',

                       'critic_score', 'user_score', 'year_of_release', 'n_platforms']

​

    buttons = []

    for column in columns_to_show:

      button=dict(

          args=[{'x': [data[column]]},

                {'xaxis': {'rangeslider': {'visible': True}, 'title': column}}],

          label=column,

          method='update',

      )

      buttons.append(button)

​

    menu = dict(

        buttons=buttons,

        direction='down',

    )

​

    fig.update_layout(

        updatemenus=[menu],

        xaxis_rangeslider_visible=True,

        title='Feature histograms',

    )

    return fig

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

interact(num_feature_histograms, data=widgets.fixed(data), years=years_slider);

def corr(data, years):

    data = data.query('@years[1] >= year_of_release >= @years[0]')

    return px.imshow(data.corr(), text_auto='.2f', title='Correlation matrix')

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

interact(corr, data=widgets.fixed(data), years=years_slider);

def sales_by_region(data, years):

    region_sales = (data

                       .query('@years[0] <= year_of_release <= @years[1]')

                       .melt(value_vars=['na_sales', 'eu_sales', 'jp_sales', 'other_sales'],

                             var_name='region', value_name='sales')

                       .replace({'na_sales': 'North America', 'eu_sales': 'European Union',

                                 'jp_sales': 'Japan', 'other_sales': 'Others'}))

    fig = px.pie(region_sales, names='region', values='sales')

    fig.update_traces(textposition='inside', textinfo='label+percent')

    fig.update_layout(title='Total sales by region', showlegend=False)

    return fig

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

interact(sales_by_region, data=widgets.fixed(data), years=years_slider);

top_platforms = data.pivot_table(index='platform', values='total_sales', aggfunc='sum')

top_platforms = list(top_platforms.sort_values(by='total_sales', ascending=False).index)

def feature_scatterplots(data, years, platform=None):

    data = data.query('@years[1] >= year_of_release >= @years[0]')

    if platform:

        data = data.query('platform == @platform')

​

    fig = px.scatter(data, x='user_score', y='total_sales')

    x_columns = ['user_score', 'critic_score', 'year_of_release', 'n_platforms',

                 'total_sales', 'na_sales', 'eu_sales', 'jp_sales', 'other_sales']

    y_columns = ['total_sales', 'na_sales', 'eu_sales', 'jp_sales', 'other_sales',

                 'critic_score', 'user_score', 'year_of_release', 'n_platforms']

​

    menus = []

    for axis, y_pos, columns_to_show in zip(['x', 'y'], [0.95, 0.8], [x_columns, y_columns]):

      buttons = []

      for column in columns_to_show:

        button=dict(

            args=[{axis: [data[column]]},

                  {f'{axis}axis': {'title': column}}],

            label=column,

            method='update',

        )

        buttons.append(button)

​

      menu = dict(

          buttons=buttons,

          direction='down',

          x=-0.14,

          y=y_pos,

      )

      menus.append(menu)

​

    fig.update_layout(

        updatemenus=menus,

        title='Feature scatterplots',

        annotations=[dict(text='x axis:', x=-0.3, y=1.02, xref='paper', yref='paper', showarrow=False),

                     dict(text='y axis:', x=-0.3, y=0.86, xref='paper', yref='paper', showarrow=False)],

    )

​

    return fig

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

interact(feature_scatterplots, data=widgets.fixed(data), years=years_slider, platform=[None] + top_platforms);

def sales_by_scores(data, years, sales):

    data = data.query("@years[0] <= year_of_release <= @years[1]")

    fig = make_subplots(1, 2, y_title=sales)

    fig.add_traces(px.scatter(data, x='user_score', y=sales).data, 1, 1)

    fig.add_traces(px.scatter(data, x='critic_score', y=sales).data, 1, 2)

    fig.update_layout(title_text='Sales by scores', title_x=0.5)

    fig.update_xaxes(title='User Score', row=1, col=1)

    fig.update_xaxes(title='Critics Score', row=1, col=2)

    return fig

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

sales = ['total_sales', 'na_sales', 'eu_sales', 'jp_sales', 'other_sales']

interact(sales_by_scores, data=widgets.fixed(data), years=years_slider, sales=sales);

def binned_scatterplot(data, years, x, y, platform, statistic, bins=10, trendline='lowess'):

    data = data.query('@years[1] >= year_of_release >= @years[0]')

    if platform:

        data = data.query('platform == @platform')

    data = data.dropna(subset=[x, y])

    x_vals = data[x]

    y_vals = data[y]

    mean_y, bin_edges, bin_numbers = stats.binned_statistic(x_vals, y_vals, statistic, bins=bins)

    std_y, bin_edges, bin_numbers = stats.binned_statistic(x_vals, y_vals, 'std', bins=bins)

    fig = px.scatter(x=(bin_edges[:-1] + bin_edges[1:]) / 2, y=mean_y, error_y=std_y, trendline=trendline)

    fig.update_layout(xaxis_title=x, yaxis_title=y, title='Feature binned scatterplots')

    if trendline:

        fig.data[1].update(line_color='red') 

    return fig

x_columns = ['user_score', 'critic_score', 'year_of_release', 'n_platforms',

                 'total_sales', 'na_sales', 'eu_sales', 'jp_sales', 'other_sales']

y_columns = ['total_sales', 'na_sales', 'eu_sales', 'jp_sales', 'other_sales',

             'critic_score', 'user_score', 'year_of_release', 'n_platforms']

years_slider = widgets.IntRangeSlider(value=[2013, 2016], min=1980, max=2016, continuous_update=False)

interact(binned_scatterplot, data=widgets.fixed(data), x=x_columns, y=y_columns, years=years_slider,

         platform=[None] + top_platforms, statistic=['mean', 'sum', 'median'],

         bins=(1, 20), trendline=['lowess', 'ols', None]);